package com.tianpengtech.common.webmagic;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.jfinal.kit.JsonKit;
import com.jfinal.plugin.activerecord.Record;

public class MockDataSocialPageProcessor {
	
	public static void main(String[] args) {
		Document doc;
		List<Record> news = new ArrayList<Record>();
		try {
			for(int i=0;i<10;i++){
				doc = Jsoup.connect("http://wei.sohu.com/roll/index_"+(i+851)+".shtml").get();
				Elements elements = doc.select("div.f14list>ul>li>a");
				for (Element element : elements) {
					Record r = new Record();
					r.set("title", element.attr("href"));
					r.set("content", element.text());
					news.add(r);
				}
			}
			System.out.println(JsonKit.toJson(news));
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
